#!/usr/bin/python
import urllib2, sqlite3

ones = ['one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight',
	'nine']
teens = ['ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 
		'sixteen', 'seventeen', 'eighteen', 'nineteen']
tens = ['twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty']

def write_file(title, passage):
	file_name = "dt/%s.txt" % title
	fd = open(file_name, 'w+')
	fd.write(passage)
	fd.close()

def fix_line(line):
	s = line
	s = s.replace("<br>", "")
	s = s.replace("&nbsp;", " ")
	if s.strip() == "":
		s = ""
	return s

def process_file(file_name):
	fd = urllib2.urlopen(file_name)
	start = False
	passage = ""
	title = ""
	last_line = ""
	for line in fd.readlines():
		if line.strip() == "<font size=\"2\"><b>":
			start = True
		elif line.strip() == "<br><br>" and title == "":
			title = last_line
		elif start == True:
			if line.strip() == "</font>":
				start = False
			else:
				passage += fix_line(line)
		last_line = line.strip()
	fd.close()
	return (passage, title)

build_list = []
build_list += ones
build_list += teens
for prefix in tens:
	build_list.append(prefix)
	build_list += list(map(lambda x: "%s%s" % (prefix, x), ones))
build_list = build_list[:-8]

db = sqlite3.connect('tao.db')
cur = db.cursor()
cur.execute("create table passages (title varchar, passage varchar)")
cur.close()
db.commit()
for prefix in build_list:
	print "Processing %s" % prefix
	file_name = "http://www.daily-tao.com/tao/%s.html" % prefix
	(passage, title) = process_file(file_name)
	cur = db.cursor()
	cur.execute("insert into passages (title, passage) values (?, ?)",
		(title, passage))
	cur.close()

db.commit()
